# encoding:utf-8
import re
from lxml import etree
from Crypto.Cipher import AES
from Crypto.Util.Padding import pad
import base64,json,math
import pandas as pd
import execjs

def get_content(html, xpath):
    if isinstance(html, str):
        html = etree.HTML(html)

    try:
        if xpath != '':
            text = html.xpath(xpath)[0]
        else:
            text = html
    except Exception:
        print('为匹配到详情页')
        return ''

    content = etree.tostring(text, encoding='utf-8', pretty_print=True, method='html').decode()
    content = re.sub('\r|\n|\t', '', content)
    content = re.sub('<img src=[\'"][/\.].*?>|\u3000|\\\\[a-z]\d{4}|？|&nbsp;|\\\\n|</?html.*?>|<title.*?title>|<head.*?head>|<meta.*?>|</?body.*?>|<style.*?style>|<script.*?script>|<link'
                     '.*?>|</?font.*?>|</?a.*?>|</?span.*?>|</?input.*?>|</?o:p>|<\?.*?>|<!--.*?-->|\r|\n|\t|\s{2,'
                     '}|</?SPAN.*?>', '', content)
    content = re.sub('\s((?!colspan|rowspan|src|\s).)*[=|:]".*?"|\xa0|\s((?!colspan|rowspan|src|\s).)*[=|:]\'.*?\'', '', content)
    content = re.sub('</p.*?>|</textarea.*?>|</h\d.*?>', '</div>', content)
    content = re.sub('<p.*?>|<textarea.*?>|<h\d.*?>', '<div>', content)
    if len(content) < 100:
        print('详情页太短')
        return ''
    return content

def aes_encrypt(str,aes_mode,key,pad_mode):
    aes2 = AES.new(key.encode('utf-8'), aes_mode)
    encrypt_text = aes2.encrypt(pad(str.encode('utf-8'), AES.block_size, style=pad_mode))
    encrypt_text = base64.encodebytes(encrypt_text).decode().replace("\n", "")
    return encrypt_text

def aes_decrypt(str,key,iv,mode):
    text = base64.b64decode(str)
    cipher = AES.new(key=key.encode(), mode=mode, IV=iv.encode())
    de_text = cipher.decrypt(text)
    unpad = lambda s: s[:-ord(s[len(s) - 1:])]
    return unpad(de_text).decode('utf-8')

import os,sys,re,shutil,time

project_templates_dir = sys.executable.replace('python.exe',r'Lib/site-packages/scrapy/templates/project/module')
spiders_templates_dir = sys.executable.replace('python.exe',r'Lib/site-packages/scrapy/templates/spiders/')

def clsname(str):
    return ''.join([i[0].upper()+i[1:]  for i in str.split('_')])

def add_spiders_templates(file_dir,spiders_templates_name):
    '''
    新增spider模板
    :param file_dir: 要新建模板的spider文件路径
    :param spiders_templates_name: 模板名称
    :return:
    '''
    project_name = re.findall('.*[\\\\/](.*?)[\\\\/]spiders',file_dir)[0]
    project_name_ = clsname(project_name)
    class_name = re.findall('.*[\\\\/](.*?)\.py',file_dir)[0]
    class_name = clsname(class_name)+'Spider'
    with open(file_dir,'r',encoding='utf-8') as f:
        text = f.read()
    name = re.findall('\sname\s*=\s*[\'"](.*?)[\'"]',text,re.S)[0]
    domain = re.findall('allowed_domains\s*=\s*\[[\'"](.*?)[\'"]\]',text,re.S)[0]
    start_url = re.findall('start_urls\s*=\s*\[[\'"]https?://(.*?)[\'"]\]',text,re.S)[0]
    text = text.replace(start_url,'$domain').replace(domain,'$domain').replace(name,'$name').replace(class_name,'$classname').replace(project_name,'$project_name').replace(project_name_,'${ProjectName}')
    new_templates_name = spiders_templates_dir+spiders_templates_name+'.tmpl'
    with open(new_templates_name,'w',encoding='utf-8') as f:
        f.write(text)
    print(f'写入成功，查看{new_templates_name}\n使用方法：scrapy genspider -t {spiders_templates_name} spidername allowed_domains')

def update_project_templates(file_dir):
    '''
    上传自己模板，省去每次startprojec都要去设置setting，pipelines，middlewares，原有模板会保存在原路径下’project buckup+时间戳’文件夹
    :param file_dir: 项目目录
    :return:
    '''
    pys = ['items.py','middlewares.py','pipelines.py','settings.py',]
    shutil.copytree(project_templates_dir,project_templates_dir.replace('project',f'project buckup{time.time()}'))
    project_name = list(filter(None,re.split('[\\\\/]',file_dir)))[-1]
    project_name_ = clsname(project_name)
    for fil in [os.path.join(file_dir, file) for file in os.listdir(file_dir)]:
        if fil.endswith('.py') or fil.endswith('.txt'):
            shutil.copy(os.path.join(file_dir,fil),os.path.dirname(project_templates_dir))
    file_dir = file_dir+project_name if file_dir.endswith('/') or file_dir.endswith('\\') else  file_dir+'/'+project_name
    file_lis = [os.path.join(file_dir, file) for file in os.listdir(file_dir)]
    for fil in file_lis:
        if fil in pys:
            with open(fil,'r',encoding='utf-8') as f:
                text = f.read()
            text = text.replace(project_name,'$project_name').replace(project_name_,'${ProjectName}')
            with open(project_templates_dir+'/'+list(filter(None,re.split('[\\\\/]',fil)))[-1]+'.tmpl','w',encoding='utf8') as f:
                f.write(text)
    print(f'写入成功，查看{project_templates_dir}')

import hashlib
def md5_encrypt(str):
    return hashlib.md5(str.encode(encoding='UTF-8')).hexdigest()

from redis import Redis
from bid.settings import RedisInit
rds = Redis(RedisInit)
def redis_dupefilter(item):
    print(md5_encrypt(item['link']))
    if not rds.sadd('link', md5_encrypt(item['link'])):
        return 1
    else:
        return 0

def search_table(items):
    ex = {
        'project_code': '(?:标[的段]（?包?）?|工程|项目|分包|采购条?目?|招标计?划?)编号',
        'succ_bidder_person': '^投标商名称|^(?:第.|成交|中标)候选人|(?:拟?中标|投标|成交)单位(?!履约|联系|电话|地址)|^中选机构名称|'
                              '^投标人(?:名称|单位名?称?)$|^第?一?中标候?选?人名?称?(?!排序|代码|.{8,})|^中标（成交）单位名称|'
                              '^成交单位名?称?|^中标人信息|(?:中标|成交)供应商名?称?(?!组织机构代码|注册|.{8,})|'
                              '^供应商名?称?(?!组织机构代码|注册|地址|承诺)|中选承包商',
        'price': '总?投资额(?!.*\d)|(?:品目|采购|)预算.{,6}|(?:合同包?|成交|预算|合[同计]|拟?中标.*?|计划|谈判|)总?金额.{,6}|'
                 '(?:成交|中标)结果(?!公告)|总价.{,6}|(?:投标|预?中标.*?|成交|标段|最高限|招标控制|合同|发包|评标)估?算?报?价格?.{,6}|预算控制额度.{,6}|'
                 '工程建设项目规模|总?报价(?!得分|百分比)|^价格$',
        'zhaobiao_person': '^(?:招标|采购|建设)单位.{,6}|^(?:招标|采购)人名?称?[:：]?$',
    }
    head_ex = '|'.join(
        ex.values()) + '|情况|资质|成员|受理|姓名|质量|承诺|审查|意见|专业类别|供应商|报价(?!百分比为)|中标人|说明|基础收费|次数|3P\(元\)|有效期|内容|比例|描述|份额|单价|控制价|价格|折扣|制造商|描述|周期|合计|' \
                       '含税价格|意向|项目奖项|工本费|保证金|法人代表|有限期|代表人|金额|包干率|机构|部门|详见|期限|资格|地址|组织机构|' \
                       '工作量|签订|联系|得分|评分|系数|标准|\d+(?:个?日?历?天|年|个?（?月）?)|监理(?!.*公司)|' \
                       '\d{4}[-年]\d{2}[-月]\d{2}|得分|评分|\d+[分家名]|备注|标书|费率|转让|是否|评审(?!中心)|规格|序号|数量|位置|' \
                       '面积|时间|工程师|文件要求|评审专家|理由|原因|项目经理|招租|（个）|第.[名中]|日至|排序|代码|初审|候选人|赋码|规模|' \
                       '总分|方式|获奖|业绩|推荐|结果|数量|^.{,10}%|下浮率|时间|审批|技术分|标段|项目名称|限价|最高|货物|名称|交货|地点|备注|售价|^代?建?单位|' \
                       '编号|日期|招标人|工期|负责人|中标项目|类型|资格|地.*?址|代理|联系人|电话|包号|型号|类别'
    text = items['content'].replace('th', 'td').replace("'\\\"1\\\"'", '').replace('美元','元')
    text = re.sub('\\\\|\'|</?tbody.*?>|</?thead.*?>|</?tdead>', '', text)
    try:
        t_list = pd.read_html(text, match='|'.join(ex.values()))
    except Exception as e:
        # print('表格匹配错误：', e)
        return items
    for t in t_list:
        t.fillna('', inplace=True)
        t = t.replace({'/|\s': ''}, regex=True)
        if len(t) == 1 or len(t.columns) == 1:
            continue
        # print(t)
        for i in range(len(t)):
            for k, v in ex.items():
                if k not in items.keys() or items[k] == '':
                    try:
                        bool_li = t.iloc[i, :].str.contains(v)
                        for j in t.iloc[i, :][bool_li].index.tolist():
                            if len(t.iloc[i, j]) > 20:
                                continue
                            try:
                                if len(re.findall(head_ex, t.iloc[i, j + 1])) == 0 and len(t.iloc[i, j + 1]) < 50:
                                    if k == 'price':
                                        try:
                                            items['price'] = re.findall('([\d\.,]+)(?!%|‰)', t.iloc[i, j + 1], re.S)[
                                                0].replace(',', '')
                                            try:
                                                items['pricetype'] = \
                                                    re.findall('([万亿元]+)', t.iloc[i, j] + t.iloc[i, j + 1], re.S)[-1][0]
                                            except:
                                                items['pricetype'] = '元'
                                            break
                                        except:
                                            pass
                                    else:
                                        if len(t.iloc[i, j + 1]) > 3:
                                            items.update({k: t.iloc[i, j + 1]})
                                            break
                            except:
                                pass
                            try:
                                if len(re.findall(head_ex, t.iloc[i + 1, j])) == 0 and len(t.iloc[i + 1, j]) < 50:
                                    if k == 'price':
                                        try:
                                            items['price'] = re.findall('([\d\.,]+)(?!%|‰)', t.iloc[i + 1, j], re.S)[
                                                0].replace(',', '')
                                            try:
                                                items['pricetype'] = \
                                                    re.findall('([万亿元]+)', t.iloc[i, j] + t.iloc[i + 1, j], re.S)[-1][0]
                                            except:
                                                items['pricetype'] = '元'
                                                if float(items['price']) < 3000 and items['pricetype'] == '元':
                                                    items['price'] = ''
                                                    items['pricetype'] = ''
                                            break
                                        except:
                                            pass
                                    else:
                                        if len(t.iloc[i + 1, j]) > 3:
                                            items.update({k: t.iloc[i + 1, j]})
                                            break
                            except:
                                pass
                    except:
                        continue
    return items

def get_field(items):
    if isinstance(items, str):
        content = items
    elif isinstance(items, dict):
        content = items['content']

    else:
        print('传入数据类型错误')
    content = re.sub('（单位章）|（盖章）|详?见公告|_{2,}|\n|&nbsp;|</?u>|\s|\\u3000|</?i>|</?em>|</strong>|<strong>|</?b>|（签名）|（[盖签]章）|项目联系方式.*?$', '', content).replace('美元','元')
    try:
        items['project_code'] = re.findall(
            '(?:项目编码[:：]|工程编号[：:]|分包编号[:：]|采购编号[：:]|项目编号[:：]|招标编号[:：]|计划编号）[:：]|项目编号</div><div>|项目编号（.*?如有）：)'
            '(.*?)(?:<|）?，|）[\u4e00-\u9fa5]|\)?,)',
            content)[-1].replace('/', '')
    except Exception:
        pass

    try:
        items['zhaobiao_person'] = \
            re.findall(
                '(?:采购单位名称：|釆购人信息.*?名称[：:]|招标人信息.*?名称[:：]|招标人及联系方式.*?单位名称[:：]|采购人信息.*?名称[:：]|项目单位[:：]|采购人名称[:：]|委托单位[:：]|采购人信息\s*名\s?称[:：]|招\s*标\s*人[:：]|采\s*购\s*人[:：]|招标人或代理机构[:：])(.*?)'
                '(?:\s|代表人|代建单位|法定代表人|组织单位|地址|代理|地点|联系|公告时间|招标|\d|日期|开户银行|采购|发布日期|技术咨询|<|\s|，)', content)[
                -1].replace('/', '')
        if len(items['zhaobiao_person']) <4:
            items['zhaobiao_person'] = ''
    except Exception:
        pass

    try:
        items['zhaobiao_contactor'] = re.findall(
            '(?:采购单位[:：]|釆购人信息.*?名称[：:]|招标人信息.*?名称[:：]|招标人及联系方式.*?单位名称[:：]|采购人信息|项目单位[:：]|委托单位[:：]|采购人信息\s*名\s?称[:：]|招\s*标\s*人[:：]|采\s*购\s*人[:：]|招标人或代理机构[:：]).*?(?:联\s*系\s*人[:：]|法定代表人[:：]|联系方式[:：]|联系人及电话[:：]|联系人：)([^1-9采购招标]*?)(?:<|\(|（|E-mail|电?子?邮箱|更多|地\s*址|法定|项目|招标|质疑|\d|\s|电\s*话|联\s*系|代理|电\s*话|，)',
            content)[-1].replace('/', '')
    except Exception:
        pass

    try:
        items['zhaobiao_phone'] = re.findall(
            '(?:采购单位|釆购人信息.*?名称[：:]|招标人信息.*?名称[:：]|招标人及联系方式.*?单位名称[:：]|采购人信息|项目单位[:：]|采购人联系人|委托单位[:：]|采购人信息\s*名\s*称[:：]|招\s*标\s*人[:：]|采\s*购\s*人[:：]|招标人或代理机构[:：]).*?(?:电话[:：]|联系方式[：:])([0|1][1-9]\d{2}-?\d{7,8})',
            content)[-1].replace('/', '')
    except Exception:
        pass

    try:
        items['zhaobiao_address'] = re.findall(
            '(?:采购单位|釆购人信息.*?名称[：:]|招标人信息.*?名称[:：]|招标人及联系方式.*?单位名称[:：]|采购人信息|项目单位[:：]|采购人地址[:：]|委托单位[:：]|采购人信息\s*名\s*称[:：]|招\s*标\s*人[:：]|采\s*购\s*人[:：]|招标人或代理机构[:：]).*?(?:地址：)(.*?)(?:<|\s|，|电话|联系人)',
            content)[-1].replace('/', '')
    except Exception:
        pass

    try:
        items['agency_danwei'] = re.findall(
            '(?:釆购代理机构信息.*?名称[：:]|招标代理机构信息.*?名称[:：]|招标代理[:：]|采购代理机构[:：]|招标代理机构[:：]|采购代理机构信息.*?名称：)(.*?)(?:<|\s|，)',
            content)[-1].replace('/', '')
        if len(items['agency_danwei']) <4:
            items['agency_danwei'] = ''
    except Exception:
        pass

    try:
        items['agency_contactor'] = re.findall(
            '(?:釆购代理机构信息.*?名称[：:]|招标代理机构信息.*?名称[:：]|招标代理机构及联系方式.*?单位名称[:：]|招标代理[:：]|采购代理机构[:：]|采购代理机构信息|招标代理机构[:：]|采购代理机构全称：).*(?:负责人[:：]|联\s*系\s*人[:：]|法定代表人[:：]|联系方式[:：]|联系人及电话[:：])([^1-9采购招标]*?)(?:<|\(|（|E-mail|电?子?邮箱|更多|地\s*址|法定|项目|招标|质疑|\d|\s|电\s*话|联\s*系|代理|电\s*话|，)',
            content)[-1].replace('/', '')
    except Exception:
        pass

    try:
        items['agency_phone'] = re.findall(
            '(?:釆购代理机构信息.*?名称[：:]|招标代理机构信息.*?名称[:：]|招标代理机构及联系方式.*?单位名称：招标代理[:：]|采购代理机构[:：]|采购代理机构信息|招标代理机构[:：]|采购代理机构联系人、联系方式：).*(?:电话[:：]|联系方式：)([\d\-、]*)(?:<|\s|，)',
            content)[-1].replace('/', '')
    except Exception:
        pass

    try:
        items['agency_address'] = re.findall(
            '(?:釆购代理机构信息.*?名称[：:]|招标代理机构信息.*?名称[:：]|招标代理机构及联系方式.*?单位名称：招标代理[:：]|采购代理机构[:：]|采购代理机构信息|招标代理机构：).*(?:地址[:：]|地址：)(.*?)(?:<|\s|，)',
            content)[-1].replace('/', '')
    except Exception:
        pass
    try:
        items['succ_bidder_person'] = re.findall(
            '(?:中标候选人第1名[：:]|(?:第一|中标（?成?交?）?)候选人名?称?[:：]|中标（成交）单位名称[:：]|成交单位名称[:：]|中标人信息[:：]|中标人名称[:：]|中标人['
            ':：]|中标候选人第 1 名 [:：]|成交供应商[:：]|供应商名称：)(.*?)(?:<|地址|，|。|统一社会|；|中标|\s|\d{10,})',
            content)[-1].replace('/', '')
    except Exception:
        pass

    try:
        price = re.findall(
            '(?:估算价|投资额|控制价|（?成交）?价|投资金?额|招标控制价|中标值|(?:最高|总)报价|中标[标总]?价|概算造价|合同金额|中标价格|总投资为|成交价格|总预算为?|预算资金|最高限价|成交价|最终报价|预算金额|中标金额（含税）?|成交[总报]价|中标价格?|中标金额为?|投标[报限]价|合同估算价?|总投资约?|投资估算|项目概算|概算投资额?|成交）金额|中标结果(?!公[示告])|成交金额|成交结果(?!公[示告])|中标金额)([^<]*?\d+.*?)'
            '(?:<|。|；|，|手机|电话|微信|邮箱|联系)',
            content)[0].replace(',','').replace('圆','元')
        print(price)
        try:
            pricelis = re.findall('([\d\.,]+)',price)
            items['price'] = str(max([float(i) for i in pricelis]))
            try:
                items['pricetype'] = re.findall('[万亿元]+',price)[-1][0]
            except:
                items['pricetype'] = '元'
        except:
            pass
        if items['pricetype'] == '元' and float(items['price'])<3000:
            items['price'] = ''
    except Exception:
        pass
    if '<table' in content:
        items.update(search_table(items))

    return items
from  scrapy_redis import dupefilter

if __name__ == '__main__':
    rds.srem('link','b0377f22a412f4dbed2070acba08b6c7')
